#!/usr/bin/python
#coding=utf-8
import logging
import urlparse
import sys
import datetime
from lxml import etree
import re
import os
from math import ceil
from PostGetter import PostGetter,htmlentitydecode,chkLogin


class QbqPostGetter(PostGetter):
	def __init__(self,cookie_file):
		PostGetter.__init__(self,cookie_file)
		self.dft_html_encoding='utf-8'
		self.NR_REPLY_PER_PAGE=20

	def login(self):
		self.logger.debug('to login ...')
		data={}
		forum=Forum.objects.filter(name=self.forum_name).get()
		if forum:
			data['username']=forum.usr.encode(self.dft_html_encoding)
			data['password']=forum.pwd.encode(self.dft_html_encoding)
			data['expires']=43200
			data['answer']=''
			data['login']=''
			data['question']=0
			data['templateid']=0
			r,_,_=self._getResponse(forum.login_url.encode(self.dft_html_encoding),data)
			if not r:
				self._signed_in=False
			else:
				r=self._tryDecode(r)
##				open('/home/kevin/tmp.html','w').write(r)

				# check html or url
				if r.find('登录成功, 返回登录前页面')!=-1:
					self.cj.save(self.cookie_file)
					self._signed_in=True
		else:
			self.logger.debug('can\'t find forum %s, can\'t do login !',self.name)

		return self._signed_in

	@chkLogin
	def getPostList(self,forum_name,sector_name,time_since,page_start,page_stop):
##		self.forum_name=forum_name
		ptoppic=re.compile('.+?t_top\d+.gif|announcement.gif'.decode(self.dft_html_encoding),re.S|re.I|re.U)
		info, debug= self.logger.info, self.logger.debug

		upd=None
		cnt=0
		try:
			forum=Forum.objects.filter(name=forum_name)[0:1].get()
		except Forum.DoesNotExist:
			debug('can\'t find forum info of %s!',forum_name)
			raise

		try:
			sector=Sector.objects.filter(forum=forum,name=sector_name)[0:1].get()
		except Sector.DoesNotExist:
			debug('can\'t find sector info of %s!',sector_name)
			raise

		for i in xrange(page_start,page_stop): # max page number
			pageurl=sector.url%(i,)
##			self.logger.info('get %s ...',pageurl)
			data=self._getData(pageurl,None,'%s-page %02d'%(sector.name,i))


			# 用 lxml xpath 选择包含正文的 div
			parser=etree.HTMLParser()
			tree=etree.fromstring(data,parser)
			el=tree.xpath('/html/body/div[@class="wrap cl"]/div[@class="main thread"]/form[@id="moderate"]/div[@class="threadlist"]/table/tbody/*')


			for item in el: # 列表中的每个主题
##				self.logger.debug('-='*10)
				if item.xpath('../@class') and item.xpath('../@class')[0]=='separation': # skip separation line
					continue

				tdlist= item.xpath('*')
				assert len(tdlist)==6
				try:
					title=tdlist[2].xpath('a/text()')[0]
				except IndexError:
					try:
						title=tdlist[2].xpath('a/font/text()')[0]
					except IndexError:
						title=tdlist[2].xpath('a/span/text()')[0]

				# 取置顶信息
				istoptopic=False
				try:
					toppic=tdlist[0].xpath('a/img/@src')[0]
				except IndexError:
					toppic=tdlist[0].xpath('img/@src')[0]
				if ptoppic.search(toppic):
					istoptopic=True

				# 取主题信息
				title_url=tdlist[2].xpath('a/@href')[0]
				m=re.search('topicid=(\d+)&?',title_url)
				if m:
					lid=m.group(1)
				else:
					debug('can\'t find locate_id for post %s',title)
					lid=''

				author=tdlist[3].xpath('cite/a/text()')[0]
				crt_date=tdlist[3].xpath('em/text()')[0]
				if crt_date.find('前')!=-1 or crt_date.find('天')!=-1:
##					debug('%s not real crt_date: %s', lid, crt_date)
					crt_date=self.str2time(crt_date).strftime('%Y-%m-%d %H:%M:%S')
				if crt_date.find('刚才')!=-1:
					crt_date=datetime.datetime.today().strftime('%Y-%m-%d %H:%M:%S')
				if crt_date.find('/')!=-1: # in form 2011/7/24 9:16:34
					crt_date=datetime.datetime.strptime(crt_date,'%Y/%m/%d %H:%M:%S').strftime('%Y-%m-%d %H:%M:%S')


				try:
					nr_reply=int(tdlist[4].xpath('a[1]/text()')[0])
				except IndexError:
					nr_reply=0

				try:
					upd_date=tdlist[5].xpath('em/a/text()')[0].strip()
				except IndexError:
					upd_date=crt_date # 找不到最近更新时间则将发表时间作为最近更新时间

				if upd_date.find('前')!=-1 or upd_date.find('天')!=-1:
##					debug('%s not real upd_date: %s', lid, upd_date)
					upd_date=self.str2time(upd_date).strftime('%Y-%m-%d %H:%M:%S')
				if upd_date.find('刚才')!=-1:
					upd_date=datetime.datetime.today().strftime('%Y-%m-%d %H:%M:%S')
				pages= int(ceil((nr_reply+1)/float(self.NR_REPLY_PER_PAGE)) )
				info('%s\n%02d)\t%s|%s|%d-%d|%s|%s', '-='*10, cnt, lid, upd_date, nr_reply, pages, author, title.strip())
				cnt+=1
				try:
					upd=datetime.datetime.strptime(upd_date,'%Y-%m-%d %H:%M:%S')
				except ValueError:
					try:
						upd=datetime.datetime.strptime(upd_date,'%Y-%m-%d %H:%M')
					except ValueError:
						upd=datetime.datetime.strptime(upd_date,'%Y/%m/%d %H:%M:%S')
#-#						upd_date=upd.strftime('%Y-%m-%d %H:%M:%S')
##				info('(%02d) (%02d) %s |%s|%s|%s|%s|%s',cnt,pages,title,author,crt_date,upd_date,nr_reply,title_url)
#-#				info('%d---%02d)%s|%s|%d-%d|%s|%s',item.sourceline,cnt,lid,upd_date,nr_reply,pages,author,title)
				if upd<time_since and (not istoptopic):
					info('%s\ngot time stop in page list.','-~'*20)
					return

				# check DB, insert if not exist, else check update
				try:
					p=Post.objects.filter(sector=sector,locate_id=lid)[0:1].get()
				except Post.DoesNotExist:
					p=Post(sector=sector, locate_id=lid, url=title_url, title=title, author=author,
						crt_date=crt_date, upd_date=upd_date, nr_reply=0)
					p.save()
					p.nr_reply=nr_reply
					debug('post %s created.',lid)
					self.getPost(p)
				else:
					if p.upd_date!=upd:
						p.upd_date=upd
						p.save()
					if p.nr_reply<nr_reply:
						info('post %s nr_reply changed. %+d',lid,nr_reply-p.nr_reply)
						self.getPost(p,nr_reply)

	def getPost(self,post,new_nr=0):
		cnt=0
		sr=urlparse.urlsplit(post.url)
		if not sr.scheme:
##			posturl=urlparse.urljoin(post.sector.base_url,post.url)
			posturl=urlparse.urljoin(post.sector.base_url, 'showtopic.aspx?topicid=%s&page=%%d'%post.locate_id)
		else:
			posturl=post.url

		startpage,stoppage=self.getPageRange(self.NR_REPLY_PER_PAGE,post.nr_reply,new_nr)

		self.logger.debug('page range [%d,%d) for post %s',startpage,stoppage,post.locate_id)
		parser=etree.HTMLParser()
		for pg in xrange(startpage,stoppage):
			self.logger.debug('post %s %s ...',post.locate_id,posturl%pg)
			data=self._getData(posturl%pg,None,post.title)
			tree=etree.fromstring(data,parser)

			pl=tree.xpath('//div[@id="postsContainer"]/table')

			for p in pl:
				if p.xpath('@class') and p.xpath('@class')[0]=='plh':
					realtitle=p.xpath('./tbody/tr/td[@class="posttopic"]/h1[@class="ts z"]/span/text()')[0]
					if realtitle and post.title!=realtitle :
						self.logger.debug('realtitle for post %s|%s',post.locate_id,realtitle)
						post.title=realtitle

					continue

				author=p.xpath('./tbody[1]/tr[1]/td[@class="postauthor"]/div[@class="poster"]/span/text()')[0]
				replyid=p.xpath('./@id')[0]
				if p.xpath('//div[@id="message%s"]/div[@id="firstpost"]'%replyid):
					replycontent=htmlentitydecode(etree.tostring(p.xpath('//div[@id="message%s"]/div[@id="firstpost"]'%replyid)[0])).strip()
				else:
					replycontent=htmlentitydecode(etree.tostring(p.xpath('//div[@id="message%s"]'%replyid)[0])).strip()
				replycontent=self.exclude_first_div_tag.match(replycontent).group(1).strip()

##				crt_date=p.xpath('./tbody[1]/tr[1]/td[@class="postcontent"]/div[@class="pi"]/div[@class="postinfo"]/em/span/@title')[0]
				crt_date=p.xpath('//div[@class="postinfo"]/em/span/@title')[0]

				try:
					r=Reply.objects.filter(post=post, author=author, locate_id=replyid)[0:1].get()
				except Reply.DoesNotExist:
					r=Reply(post=post,locate_id=replyid,crt_date=crt_date,author=author,content=replycontent)
					r.save()
					cnt+=1

##		debug('post %s add %d reply.',post.locate_id,cnt)
		if new_nr!=0:
			if post.nr_reply+cnt==new_nr:
				self.logger.info('post %s %+d reply. now %d',post.locate_id,cnt,new_nr+1)
				post.nr_reply+=cnt # 增加实际变化（新增）的数量
			else:
				self.logger.debug('post %s %+d reply, %d != expect %d',post.locate_id,cnt,post.nr_reply+cnt,new_nr)
				# 检查实际获得数量
				actualcnt=Reply.objects.filter(post=post).count()
				self.logger.info('post %s actual %d reply in DB',post.locate_id,actualcnt)
				post.nr_reply=actualcnt-1 if actualcnt-1>=0 else 0
		else:
			if post.nr_reply+1==cnt:
				self.logger.info('post %s init %+d reply.',post.locate_id,cnt)
			else:
				self.logger.info('post %s init %+d reply, != expect %d',post.locate_id,cnt,post.nr_reply+1)
			post.nr_reply=cnt-1 if cnt-1>=0 else 0 # 设为实际获得值-1，以便下次再次尝试查找新增帖子
		post.save()

	def str2time(self,strtime):
		p=re.compile('(\d+)小时|(\d+)分钟|(\d+)秒前'.decode(self.dft_html_encoding),re.U)
		m=p.search(strtime)
		if m:
			if m.group(1):
				return datetime.datetime.today()-datetime.timedelta(hours=int(m.group(1)))
			elif m.group(2):
				return datetime.datetime.today()-datetime.timedelta(minutes=int(m.group(2)))
			elif m.group(3):
				return datetime.datetime.today()-datetime.timedelta(seconds=int(m.group(3)))

		p=re.compile('昨天 (\d+):(\d+)|前天 (\d+):(\d+)|今天 (\d+):(\d+)'.decode(self.dft_html_encoding),re.U)
		m=p.search(strtime)
		if m:
			if m.group(1) and m.group(2):
				x=datetime.datetime.today()-datetime.timedelta(days=1)
				return datetime.datetime.strptime('%d-%02d-%02d %02d:%02d'%(x.year,x.month,x.day, int(m.group(1)), int(m.group(2))),'%Y-%m-%d %H:%M')
			elif m.group(3) and m.group(4):
				x=datetime.datetime.today()-datetime.timedelta(days=2)
				return datetime.datetime.strptime('%d-%02d-%02d %02d:%02d'%(x.year,x.month,x.day, int(m.group(3)), int(m.group(4))),'%Y-%m-%d %H:%M')
			elif m.group(5) and m.group(6):
				x=datetime.datetime.today()
				return datetime.datetime.strptime('%d-%02d-%02d %02d:%02d'%(x.year,x.month,x.day, int(m.group(5)), int(m.group(6))),'%Y-%m-%d %H:%M')

		self.logger.exception('\'can\'t process strtime %s ', strtime)

		return datetime.datetime.today()



os.environ['DJANGO_SETTINGS_MODULE']='postgetter.settings'
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
# 重要 为了能使用django的orm模块，需要创建目录btView，在里面放上空
#  文件__init__.py和文件models.py,这样django就会去数据库中找btView_Video,
#  btView_Screenshot，btView是创建的另一个ajango应用的app_name
# 如何单独使用ajango的orm模块见 http://wiki.woodpecker.org.cn/moin/UsingDjangoAsAnStandaloneORM
from postgetter.getpost.models import Forum,Sector,Post,Reply
from django.db import transaction

if __name__=='__main__':
	reload(sys)
	sys.setdefaultencoding('utf-8')

	# django will set env['TZ'] according to the settings.py of django project
	logging.info('Time Zone: %s',os.environ['TZ']) #	os.environ['TZ']='Asia/Shanghai'

	pg=QbqPostGetter(os.path.join(os.path.dirname(os.path.abspath(__file__)),'qbq_cookie.dat'))
	pg.forum_name='qbq'
	pg.doWork('qbq','qbq-茶馆',
##								datetime.datetime.today()-datetime.timedelta(minutes=60),
##								datetime.datetime.today()-datetime.timedelta(hours=25),
								datetime.datetime.today()-datetime.timedelta(days=5),
								1,10)

	pg.doWork('qbq','qbq-警务',
##								datetime.datetime.today()-datetime.timedelta(minutes=60),
##								datetime.datetime.today()-datetime.timedelta(hours=25),
								datetime.datetime.today()-datetime.timedelta(days=5),
								1,10)
	logging.debug('done')
	raw_input('press enter to exit ...')
